#-*- coding:utf-8 -*-
import re
import time
import string
import sys
import os
import hashlib
import urllib
import urllib2
from bs4 import BeautifulSoup
import requests
from lxml import etree
from weibohelper import DbUtils
import traceback
from proxyhelper import Proxy_DbUtils
import ConfigParser
import json
class myproxy:
    def __init__(self):
        print 'myproxy init...'

    def getIp(self, i):
        #r = requests.get('http://127.0.0.1:8000/?types=0&count=30&country=国内')
        #ip_ports = json.loads(r.text)
        ip_ports = Proxy_DbUtils().queryProxy()
        ip = ip_ports[i][0]
        port = ip_ports[i][1]
        proxies = {
            'http': 'http://%s:%s' % (ip, port),
            'https': 'http://%s:%s' % (ip, port)
        }
        return proxies


    def weibo_check(self, proxies, url, cookies):
        flag = False
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.8',
            'Connection': 'keep-alive',
            'Accept-Encoding': 'gzip, deflate',
        }
        try:
            #r = requests.get(url=url, headers=headers, timeout=3,
            #                proxies=proxies)
            #r.encoding = chardet.detect(r.content)['encoding']

            html = requests.get(url=url, cookies=cookies, headers=headers, proxies=proxies, timeout=3).content
            if html.strip() == "":
                flag = False
            else:
                flag = True
        except Exception as e:
            flag = False
            html = ""

        return flag, html


class weibo:
    #proxy_i = 1
    first_time = True
    cookie = { "Cookie": "SCF=AlnE5jaK7xjdG9AxuHx3xAprP_D5mWbcJyxa7r7rZs1Dppi0-nht0EC7aLWd2T2gpY3MStH0SbNT7TJgjifekTI.; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WFMU-CyUqgxpa7s1VnzVKZx5JpX5o2p5NHD95Q0eoBpeonc1K.NWs4Dqcjwi--ciK.Ni-27i--4i-2EiK.R1K.0eoBpeonc1K.N; _T_WM=c4eb91b320e3932decf6f1a967403ee7; SUB=_2A250a2rWDeRhGeVM71MT-CjFwjuIHXVXlHaerDV6PUJbkdANLUvskW0k0ODIXTaf3eevSuQXnOgYI07ANA..; SUHB=0gHiWxzZaOVfeA; SSOLoginState=1500453510" }
    #weibo类初始化
    def __init__(self,user_id,filter = 0,tmp_cookies="",updateCycle=0,updateNum=0):
            self.user_id = user_id #用户id，即需要我们输入的数字，如昵称为“Dear-迪丽热巴”的id为1669879400
            self.filter = filter #取值范围为0、1，程序默认值为0，代表要爬取用户的全部微博，1代表只爬取用户的原创微博
            self.userName = '' #用户名，如“Dear-迪丽热巴”
            self.weiboNum = 0 #用户全部微博数
            self.weiboNum2 = 0 #爬取到的微博数
            self.following = 0 #用户关注数
            self.followers = 0 #用户粉丝数
            self.weibos = [] #微博内容
            self.num_zan = [] #微博对应的点赞数
            self.num_forwarding = [] #微博对应的转发数
            self.num_comment = [] #微博对应的评论数


            self.proxy_i = 0
            self.updateCycle = updateCycle
            self.updateNum = updateNum

    #获取用户昵称
    def getUserName(self):
      try:

        # print proxy
        url = 'http://weibo.cn/%d/info'%(self.user_id)

        '''proxy = myproxy().getIp(self.proxy_i)
        flag, html = myproxy().weibo_check(proxy, url,weibo.cookie)
        while (flag == False and self.proxy_i < 30):
              self.proxy_i += 1
              proxy = myproxy().getIp(self.proxy_i)
              flag, html = myproxy().weibo_check(proxy, url, weibo.cookie)
        print '有效代理: ' + str(proxy)'''
        html = requests.get(url, cookies = weibo.cookie,headers={'Connection':'close'}).content
        selector = etree.HTML(html)

        userName = selector.xpath("//title/text()")[0]
        self.userName = userName[:-3].encode('gbk')


        print self.userName.decode('gbk', 'ignore').encode('utf-8')
        #print '用户昵称：' + self.userName
      except Exception,e:
        print "Error: ",e
        traceback.print_exc()

    #获取用户微博数、关注数、粉丝数
    def getUserInfo(self):
      try:

        url = 'http://weibo.cn/u/%d?filter=%d&page=1'%(self.user_id,self.filter)  #,proxies=myproxy().getIp(weibo.proxy_i)

        '''proxy = myproxy().getIp(self.proxy_i)
        flag, html = myproxy().weibo_check(proxy, url, weibo.cookie)
        while (flag == False and self.proxy_i < 30):
            self.proxy_i += 1
            proxy = myproxy().getIp(self.proxy_i)
            flag, html = myproxy().weibo_check(proxy, url, weibo.cookie)
        print '有效代理: ' + str(proxy)'''

        html = requests.get(url, cookies = weibo.cookie,headers={'Connection':'close'}).content
        selector = etree.HTML(html)
        pattern = r"\d+\.?\d*"

        #微博数
        str_wb = selector.xpath("//div[@class='tip2']/span[@class='tc']/text()")[0]
        guid = re.findall(pattern, str_wb, re.S|re.M)
        for value in guid:
            num_wb = int(value)
            break
        self.weiboNum = num_wb
        #print '微博数: ' + str(self.weiboNum)
  
        #关注数
        str_gz = selector.xpath("//div[@class='tip2']/a/text()")[0]
        guid = re.findall(pattern, str_gz, re.M)
        self.following = int(guid[0])
        #print '关注数: ' + str(self.following)

        #粉丝数
        str_fs = selector.xpath("//div[@class='tip2']/a/text()")[1]
        guid = re.findall(pattern, str_fs, re.M)
        self.followers = int(guid[0])
        #print '粉丝数: ' + str(self.followers)
      except Exception,e:
        print "Error: ",e
        traceback.print_exc()

    #获取用户微博内容及对应的点赞数、转发数、评论数
    def getWeiboInfo(self):
      try:
        url = 'http://weibo.cn/u/%d?filter=%d&page=1'%(self.user_id,self.filter)  #,proxies=myproxy().getIp(weibo.proxy_i)

        '''proxy = myproxy().getIp(self.proxy_i)
        flag, html = myproxy().weibo_check(proxy, url, weibo.cookie)
        while (flag == False and self.proxy_i < 30):
            self.proxy_i += 1
            proxy = myproxy().getIp(self.proxy_i)
            flag, html = myproxy().weibo_check(proxy, url, weibo.cookie)
        print '有效代理: ' + str(proxy)'''
        html = requests.get(url, cookies = weibo.cookie,headers={'Connection':'close'}).content
        selector = etree.HTML(html)
        if selector.xpath('//input[@name="mp"]')==[]:
           pageNum = 1
        else:
           pageNum = (int)(selector.xpath('//input[@name="mp"]')[0].attrib['value'])
        pattern = r"\d+\.?\d*"

        if(weibo.first_time == True):
         weibo.first_time = False
        else:
         pageNum=updateNum
        for page in range(1,pageNum+1):
          url2 = 'http://weibo.cn/u/%d?filter=%d&page=%d' % (self.user_id, self.filter, page)
          #proxy = myproxy().getIp(self.proxy_i)
          #if(page % 5 == 0):

            #proxy = myproxy().getIp(self.proxy_i)
          #while (myproxy().weibo_check(proxy, url2) == False and self.proxy_i < 30):
          #        self.proxy_i += 1
          #        proxy = myproxy().getIp(self.proxy_i)
          '''proxy = myproxy().getIp(self.proxy_i)
          flag, html2 = myproxy().weibo_check(proxy, url2, weibo.cookie)
          while (flag == False and self.proxy_i < 30):
              self.proxy_i += 1
              proxy = myproxy().getIp(self.proxy_i)
              flag, html2 = myproxy().weibo_check(proxy, url2, weibo.cookie)
          print '有效代理: ' + str(proxy)'''
          time.sleep(5)
          html2 = requests.get(url2, cookies = weibo.cookie,headers={'Connection':'close'}).content
          selector2 = etree.HTML(html2)
          info = selector2.xpath("//div[@class='c']")
          #print len(info)
          if len(info) > 3:
            for i in range(0,len(info)-2):
              self.weiboNum2 = self.weiboNum2 + 1
              #微博内容
              str_t = info[i].xpath("div/span[@class='ctt']")
              weibos = str_t[0].xpath('string(.)').encode('gbk','ignore')
              self.weibos.append(weibos)
              #print '微博内容：'+ weibos
              #点赞数
              str_zan = info[i].xpath("div/a/text()")[-4]
              guid = re.findall(pattern, str_zan, re.M)
              num_zan = int(guid[0])
              self.num_zan.append(num_zan)
              #print '点赞数: ' + str(num_zan)
              #转发数
              forwarding = info[i].xpath("div/a/text()")[-3]
              guid = re.findall(pattern, forwarding, re.M)
              num_forwarding = int(guid[0])
              self.num_forwarding.append(num_forwarding)
              #print '转发数: ' + str(num_forwarding)
              #评论数
              comment = info[i].xpath("div/a/text()")[-2]
              guid = re.findall(pattern, comment, re.M)
              num_comment = int(guid[0])
              self.num_comment.append(num_comment)
              #print '评论数: ' + str(num_comment)

              m2 = hashlib.md5()
              m2.update(weibos)


              flag = m2.hexdigest()
              if DbUtils().queryByFlag(flag):
                  continue
              else:
                  DbUtils().insert(url2,weibos.decode('gbk', 'ignore').encode('utf-8'),flag,num_forwarding,num_zan,num_comment)

        if self.filter == 0:
          print '共'+str(self.weiboNum2)+'条微博'
        else:
          print '共'+str(self.weiboNum)+'条微博，其中'+str(self.weiboNum2)+'条为原创微博'
      except Exception,e:
        print "Error: ",e
        traceback.print_exc()

    #主程序
    def start(self):
      try:
        weibo.getUserName(self)
        weibo.getUserInfo(self)
        weibo.getWeiboInfo(self)
        print '信息抓取完毕'
        print '==========================================================================='
      except Exception,e:
        print "Error: ",e
    
    #将爬取的信息写入文件	
    def writeTxt(self):
      try:
        if self.filter == 1:
           resultHeader = '\n\n原创微博内容：\n'
        else:
           resultHeader = '\n\n微博内容：\n'
        result = '用户信息\n用户昵称：' + self.userName.decode('gbk', 'ignore').encode('utf-8') + '\n用户id：' + str(self.user_id) + '\n微博数：' + str(self.weiboNum) + '\n关注数：' + str(self.following) + '\n粉丝数：' + str(self.followers) + resultHeader
        for i in range(1,self.weiboNum2 + 1):
          text=str(i) + ':' + self.weibos[i-1].decode('gbk', 'ignore').encode('utf-8') + '\n'+'点赞数：' + str(self.num_zan[i-1]) + '	 转发数：' + str(self.num_forwarding[i-1]) + '	 评论数：' + str(self.num_comment[i-1]) + '\n\n'
          result = result + text
        if os.path.isdir('weibo') == False:
           os.mkdir('weibo')
        f = open("weibo/%s.txt"%self.user_id, "wb")
        f.write(result)
        f.close()
        file_path=os.getcwd()+"\weibo"+"\%d"%self.user_id+".txt"
        print '微博写入文件完毕，保存路径%s'%(file_path)
      except Exception,e:
        print "Error: ",e
        traceback.print_exc()





cp = ConfigParser.SafeConfigParser()
cp.read('spider.conf')

if cp.has_option('weibo', 'cookies')and cp.has_option('weibo', 'weiboId') and cp.has_option('update',  'updateCycle') and cp.has_option('update', 'updateNum'):
    tmp_cookies = cp.get('weibo', 'cookies')
    weiboId = cp.get('weibo', 'weiboId')
    updateCycle = cp.get('update', 'updateCycle')
    updateNum = cp.get('update', 'updateNum')

    print 'tep_cookies: ', tmp_cookies
    print 'weiboId', weiboId
    print 'updateCycle', updateCycle
    print 'updateNum', updateNum

    # 使用实例,输入一个用户id，所有信息都会存储在wb实例中
    user_id = int(weiboId)  # 可以改成任意合法的用户id（爬虫的微博id除外）
    filter = 0  # 值为0表示爬取全部的微博信息（原创微博+转发微博），值为1表示只爬取原创微博
    wb = weibo(user_id, filter,tmp_cookies,updateCycle,updateNum)  # 调用weibo类，创建微博实例wb
    wb.start()  # 爬取微博信息
    print '用户名：' + wb.userName.decode('gbk', 'ignore').encode('utf-8')
    print '全部微博数：' + str(wb.weiboNum)
    print '关注数：' + str(wb.following)
    print '粉丝数：' + str(wb.followers)
    print '最新一条微博为：' + wb.weibos[0].decode('gbk', 'ignore').encode(
        'utf-8')  # 若filter=1则为最新的原创微博，如果该用户微博数为0，即len(wb.weibos)==0,打印会出错，下同
    print '最新一条微博获得的点赞数：' + str(wb.num_zan[0])
    print '最新一条微博获得的转发数：' + str(wb.num_forwarding[0])
    print '最新一条微博获得的评论数：' + str(wb.num_comment[0])
    wb.writeTxt()  # wb.writeTxt()只是把信息写到文件里，大家可以根据自己的需要重新编写writeTxt()函数
else:



     i = 0

     proxy = myproxy().getIp(i)
     print proxy
     while(myproxy().weibo_check(proxy) == False and i < 30):
         i += 1
         proxy = myproxy().getIp(i)
     print myproxy().weibo_check(proxy)
     print i
     print '请查看配置是否正确'
